#include <boost/regex.hpp>
#include <string>
#include <iostream>
#include <list>
#include "HTMLParser.h"
#include "./dicionario/Timer.h"
#include "Substituter.h"

using namespace std;


struct _twoUInts{
		uint open;
		uint close;
	};

string HTMLParser::extractText(string htmlPage) {
////	cout << htmlPage << endl;
//	boost::regex newlines("\n");
//
//	htmlPage = boost::regex_replace(htmlPage,newlines, "");
//	boost::regex tags("(<script></script>)", boost::regex::icase | boost::regex::perl);
//////	if (regex_match(htmlPage, tags) ) {
//////		return "true";
//////	}
//////	else
//////		return "false";
//	string retorno = htmlPage;
//	do {
//		htmlPage = retorno;
//		retorno = boost::regex_replace(htmlPage,tags, "");
//		cout << "HTML:    " << htmlPage << endl << endl;
//		cout << "RETORNO: " << retorno << endl << endl << endl;
//	}while (htmlPage != retorno);
//	return retorno;
////    boost::regex reg("(Colo)(u)(r)",
////    boost::regex::icase|boost::regex::perl);
////
////    std::string s="Colour, colours, color, colourize";
////    s=boost::regex_replace(s,reg,"$1$3");
////    std::cout << s;
////    return "";
	
	
	
	
		toLower(htmlPage);
	if (mayIndex(htmlPage)) {
//		cout << endl <<  "745eyhbrtfdthAAAAAAAAAA****************" << endl;
		if (!isolateBody(htmlPage))
			removeHead(htmlPage);
		removeInlineTags(htmlPage);
//		cout << htmlPage << endl;
		removeScripts(htmlPage);
		removeStyles(htmlPage);
//		cout << "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" << endl;
//		cout << "Entrada: " << endl << endl << endl << endl <<"htmlPage: " << htmlPage << endl;
		removeAllTags(htmlPage);
//		cout << "*********************************************************************************" << endl;
//		cout << "Saida: " << endl << endl << endl << endl <<"htmlPage: " << htmlPage << endl;
//		uint kfdls;
//		cin >> kfdls;
		htmlPage = substituter.convertHtmlString(htmlPage, CONVERTHTMLSTRING);
		removeEnters(htmlPage);
		return htmlPage;
	}
	else {
//		cout << endl << "AAAAAAAAAAAAAAAAAAAA****************" << endl;
		return "";
	}	
}



bool HTMLParser::mayIndex (const string & str) {
	timer.begin();
	
	uint ltLocation;
    bool strFound = false;
    uint gtLocation = 0;
    string temp;
    do {
        ltLocation= str.find_first_of('<', gtLocation);
        if (ltLocation != string::npos)
        {
            gtLocation = str.find_first_of('>', ltLocation);
            if (gtLocation != string::npos) {
                temp = str.substr(ltLocation + 1, gtLocation - ltLocation - 1);
                strFound = ( (temp.find("meta") != string::npos) && (temp.find("noindex") != string::npos) );
            }
        }
        else {
        	timer.end();
        	MAYINDEX += timer.diff();
            return true;
        }
    }while(!strFound);
    
    timer.end();
    MAYINDEX += timer.diff();
    
    return false;
}

/**
 * Remove desde a primeira tag <head> ate a ultima tag </head> (caso os dois existam) nessa ordem
*/
bool HTMLParser::removeHead(string & str) {
	timer.begin();
	
	uint firstHeadOpen = str.find("<head>");
	uint lastHeadClose = str.rfind("</head>");
	if (firstHeadOpen != string::npos && lastHeadClose != string::npos) {
		str = str.substr(0, firstHeadOpen) + str.substr (lastHeadClose + 7, str.size() - (lastHeadClose + 7));
		
		timer.end();
		REMOVEHEAD += timer.diff();
		
		return true;
	}
	else {
		REMOVEHEAD += timer.diff();
		return false;
	}
	
}


/**
 * Isola <body> </body>, se houver, eliminando as tags em questao (do primeiro <body ao ultimo </body> 
 */
bool HTMLParser::isolateBody(string & str) {
	timer.begin();
	
	uint firstHeadOpen = str.find("<body");
	uint lastHeadClose = str.rfind("</body>");
	if (firstHeadOpen != string::npos && lastHeadClose != string::npos) {
		str = str.substr(firstHeadOpen, lastHeadClose + 7 - firstHeadOpen);
		
		timer.end();
		ISOLATEBODY += timer.diff();
		
		return true;
	}
	else {
		timer.end();
		ISOLATEBODY += timer.diff();
		return false;
	}	
}


/*
 * Pressupoe q nao tem </script> antes do primeiro <script
 */
void HTMLParser::removeScripts(string & str) {
	timer.begin();
	
//	uint numScripts = 0;
//	uint end = 0;
////	cout << "Achando scripts em: " << str << endl;
//	uint scriptBeginTag = str.find("<script");
//	uint scriptEndTag = str.find ("</script>");
////	cout << "blah " << scriptBeginTag << "  " << scriptEndTag << endl;
//	while (scriptEndTag != string::npos) {
////		cout << "str: " << str << endl;
////		cout << "while: " << scriptBeginTag << " "<< scriptEndTag  << " " << numScripts << endl;
//		if (scriptBeginTag < scriptEndTag) {
//			if (numScripts == 0)
//				end = scriptBeginTag;
//			numScripts++;
//			str = str.substr(0, end) + str.substr(scriptBeginTag + 8, str.size() - scriptBeginTag - 8);
//		}
//		else {
//			numScripts--;
//			if (numScripts == 0) {
//				str = str.substr(0, end) + str.substr(scriptEndTag + 9, str.size() - (scriptEndTag + 9));
//			}
//			else
//				str = str.substr(0, end) + str.substr(scriptEndTag + 9, str.size() - scriptEndTag - 9);
//		}
//		
//		scriptBeginTag = str.find("<script");
//		scriptEndTag = str.find ("</script>");
//		
////		cout << endl;
//	}
	
	
	//Tentativa 1:
	if (str.size() <= 1)
		return;
	
	uint close = str.find("</script>");
	uint numRemover = 0;
	
	vector<_twoUInts> tags;
	_twoUInts uints;
	
	uints.open = 0;
	uints.close = 0;
	
	tags.push_back(uints);
	
	while (close != string::npos) {
		bool sucesso = true;
		uint open = str.rfind("<script", close);
		if (open == string::npos)  {
			if (tags.size() != 1 + numRemover) {
				cout << "O impossivel aconteceu!!!" << endl;
				exit(473);
			}
			close = str.find("</script>", close + 1);
			continue;
		}
		else {
			while (open < tags[tags.size() - 1 - numRemover].close) {
				open = str.rfind("<script", tags[tags.size() - 1 - numRemover].open - 1);
				if (open == string::npos)  {
					sucesso = false;
					break;
				}
				numRemover++;
			}
			if (!sucesso) {
				sucesso = true;
				numRemover = 0;
				close = str.find("</script>", close + 1);
				continue;
			}
			while (numRemover > 0) {
//				cout << "Removendo: " << tags[tags.size()-1].open << " " << tags[tags.size()-1].close << endl;
				tags.pop_back();
				
				numRemover --;
			}

			
			uints.open = open;
			uints.close = close;
			tags.push_back(uints);
			//cout << "ENTRE: " << str << endl;
			close = str.find("</script>", close + 1);
		}
	}
	
//	cout << "tamanho da lista: " << tags.size() << endl;

	vector<_twoUInts>::iterator it;
	vector<_twoUInts>::iterator it2;
	string temp = "";
	temp.reserve(str.size() - tags.size());
	for (it = tags.begin(); it != tags.end(); it++) {
		it2 = it;
		it2++;
		if (it == tags.begin()) {
			if (it2 == tags.end())
				return;
			temp += str.substr(it->close, it2->open - it->close);
		}
		else if (it2 == tags.end())  {
//			cout << "c, d, size: " << it->b << ", " << str.size() - it->b << ", " << str.size() << endl;
			temp += str.substr(it->close + 9, str.size() - it->close - 9);
		}
		else {
//			cout << "a, b, size: " << it->b << ", " << it2->a - it->b << ", " << str.size() << endl;
			temp += str.substr(it->close + 9, it2->open - it->close - 9);
		}
	}
	str = temp;

	
	
	timer.end();
	REMOVESCRIPTS += timer.diff();
}

void HTMLParser::removeInlineTags(string & str) {
	timer.begin();
	
//	boost::regex tags("<[^/][^>]*/[ 	\n\r]*>");
//	boost::regex tags("<[^>]*/>");
//	str = boost::regex_replace(str,tags, " ");
//	cout << "ANTES: " << str << endl;
	
	
	
	
	//Tentativa 1:
	uint close = str.find("/>");
	
	list<_twoUInts> tags;
	_twoUInts uints;
	
	uints.open = 0;
	uints.close = 0;
	
	tags.push_back(uints);
	
	while (close != string::npos) {
		uint open = str.rfind("<", close);
		if (open == string::npos) {
			close = str.find("/>", close + 1);
			continue;
		}
//		cout << "open: " << open << " close: " << close << endl;
//		str.erase (open, close-open+2);
		uints.open = open;
		uints.close = close;
		
		tags.push_back(uints);
		//cout << "ENTRE: " << str << endl;
		close = str.find("/>", close + 1);
	}

	list<_twoUInts>::iterator it;
	list<_twoUInts>::iterator it2;
	string temp = "";
	for (it = tags.begin(); it != tags.end(); it++) {
		it2 = it;
		it2++;
		if (it2 == tags.end())  {
//			cout << "c, d, size: " << it->b << ", " << str.size() - it->b << ", " << str.size() << endl;
			temp += str.substr(it->close + 2, str.size() - it->close - 2);
		}
		else if (it == tags.begin()) {
			temp += str.substr(it->close, it2->open - it->close);
		}
		else {
//			cout << "a, b, size: " << it->b << ", " << it2->a - it->b << ", " << str.size() << endl;
			temp += str.substr(it->close + 2, it2->open - it->close - 2);
		}
	}
	str = temp;
	
	
	
//	cout << "DEPOIS: " << str << endl;
	
	timer.end();
	REMOVEINLINETAGS += timer.diff();
}

void HTMLParser::removeAllTags(string & str) {
	timer.begin();
	
//	boost::regex tags("<[^>]*>");
//	str = boost::regex_replace(str,tags, " ");	
//	trim(str);
	
	//Tentativa 1:
	
	if (str.size() <= 1)
		return;
	
	uint close = str.find(">");
	uint numRemover = 0;
	
	vector<_twoUInts> tags;
	_twoUInts uints;
	
	uints.open = 0;
	uints.close = 0;
	
	tags.push_back(uints);
	
	while (close != string::npos) {
		bool sucesso = true;
		uint open = str.rfind("<", close);
		if (open == string::npos)  {
			if (tags.size() != 1 + numRemover) {
				cout << "O impossivel aconteceu!!!" << endl;
				exit(473);
			}
			close = str.find(">", close + 1);
			continue;
		}
		else {
			while (open < tags[tags.size() - 1 - numRemover].close) {
				open = str.rfind("<", tags[tags.size() - 1 - numRemover].open - 1);
				if (open == string::npos)  {
					sucesso = false;
					break;
				}
				numRemover++;
			}
			if (!sucesso) {
				sucesso = true;
				numRemover = 0;
				close = str.find(">", close + 1);
				continue;
			}
			while (numRemover > 0) {
//				cout << "Removendo: " << tags[tags.size()-1].open << " " << tags[tags.size()-1].close << endl;
				tags.pop_back();
				
				numRemover --;
			}

			
			uints.open = open;
			uints.close = close;
			tags.push_back(uints);
			//cout << "ENTRE: " << str << endl;
			close = str.find(">", close + 1);
		}
	}
	
//	cout << "tamanho da lista: " << tags.size() << endl;

	vector<_twoUInts>::iterator it;
	vector<_twoUInts>::iterator it2;
	string temp = "";
	temp.reserve(str.size() - tags.size());
	for (it = tags.begin(); it != tags.end(); it++) {
		it2 = it;
		it2++;
		if (it == tags.begin()) {
			if (it2 == tags.end())
				return;
			temp += str.substr(it->close, it2->open - it->close);
		}
		else if (it2 == tags.end())  {
//			cout << "c, d, size: " << it->b << ", " << str.size() - it->b << ", " << str.size() << endl;
			temp += str.substr(it->close + 1, str.size() - it->close - 1);
		}
		else {
//			cout << "a, b, size: " << it->b << ", " << it2->a - it->b << ", " << str.size() << endl;
			temp += str.substr(it->close + 1, it2->open - it->close - 1);
		}
	}
	str = temp;
	timer.end();
	REMOVEALLTAGS += timer.diff();
}

void HTMLParser::removeEnters(string & str) {
	timer.begin();
	
//	boost::regex tags("[ 	\\n\\r]+");
//	str = boost::regex_replace(str,tags, " ");	
	
	
	//Tentativa 1:
	string temp = "";
	temp.reserve(str.size());
	uint lastIndex = 0;
	string delimiters = " \t\n\r";

	lastIndex = str.find_first_not_of(delimiters);
	
	if (lastIndex == string::npos) {
		str = "";
		return;
	}
	
	uint firstThing = str.find_first_of(delimiters, lastIndex);
	
	while (firstThing != string::npos) {
		if (firstThing != lastIndex) {
			temp += str.substr (lastIndex, firstThing - lastIndex) + " ";
			lastIndex = firstThing + 1;
			if (lastIndex >= str.size())
				break;
		}
		else {
			lastIndex = firstThing + 1;
		}
		firstThing = str.find_first_of(delimiters, lastIndex);

	}
//	cout << lastIndex << " " << str.size() << endl;
	if (lastIndex < str.size())
		temp += str.substr(lastIndex, str.size() - lastIndex);
	
	lastIndex = temp.size() - 1;
	if (temp[lastIndex] == ' ')
		str = temp.substr(0, temp.size() - 1);
	else
		str = temp;
	
	timer.end();
	REMOVEENTERS += timer.diff();
}

void HTMLParser::removeStyles(string & str) {
	timer.begin();
	
	uint numScripts = 0;
	uint end = 0;
//	cout << "Achando scripts em: " << str << endl;
	uint scriptBeginTag = str.find("<style");
	uint scriptEndTag = str.find ("</style>");
//	cout << "blah " << scriptBeginTag << "  " << scriptEndTag << endl;
	while (scriptEndTag != string::npos) {
//		cout << "str: " << str << endl;
//		cout << "while: " << scriptBeginTag << " "<< scriptEndTag  << " " << numScripts << endl;
		if (scriptBeginTag < scriptEndTag) {
			if (numScripts == 0)
				end = scriptBeginTag;
			numScripts++;
			str = str.substr(0, end) + str.substr(scriptBeginTag + 7, str.size() - scriptBeginTag - 7);
		}
		else {
			numScripts--;
			if (numScripts == 0) {
				str = str.substr(0, end) + str.substr(scriptEndTag + 8, str.size() - (scriptEndTag + 8));
			}
			else
				str = str.substr(0, end) + str.substr(scriptEndTag + 8, str.size() - scriptEndTag - 8);
		}
		
		scriptBeginTag = str.find("<style");
		scriptEndTag = str.find ("</style>");
		
//		cout << endl;
	}
	
	timer.end();
	REMOVESTYLES += timer.diff();
}

bool HTMLParser::checkIgnoreCase(string str, string compared) {
    trim(str);
    if (str.size() < compared.size()) return false;
    
    for (uint i = 0; i < compared.size(); i++) {
        if ( !(str[i] == compared[i]) )
            return false;
    }
    return true;
}

void HTMLParser::toLower(string & str) {
	timer.begin();

	for (uint i = 0; i < str.size(); i++)
		str[i] = tolower(str[i]);   
	
	timer.end();
	TOLOWER += timer.diff();
}

void HTMLParser::trim(string &str) {
    try{
//    cout << "antes trim: " << str << endl;
    string forbidden = " \t\r";
    while (str.find_last_of(forbidden) == str.size()-1 && str.size() > 0) {
        str.erase(str.size()-1);
//        cout << "inicio trim:  " << str << endl;
    }
    while (str.find_first_of(forbidden) == 0 && str.size() > 0) {
        str.erase(0,1);
//        cout << "meio trim:  " << str << endl;
    }
//    cout << "fim trim: " << str << endl;
    }catch(...){
        cout << "MC cagou o pau no HTML Parser " << endl;
    }
}